package com.spider;

import com.lagou.analysis.entity.Position;
import com.spider.db.DataSet;
import com.spider.util.HttpRequestUtil;
import org.json.JSONArray;
import org.json.JSONObject;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class Spider {

    private ExecutorService executorService = new ThreadPoolExecutor(5,10,1, TimeUnit.SECONDS,new LinkedBlockingDeque<>());

    public static void main(String[] args) throws Exception{
        Integer i  = 1;
        Integer totalPage = 2;

        while(i < totalPage) {
            JSONObject object = getJsonObjectByPageIndex(i);
            System.out.println("-----------  -------------------");

           if(object == null || object.equals(JSONObject.NULL) || object.getJSONObject("content").getJSONObject("positionResult").get("result").equals(JSONObject.NULL)) {
               continue;
           }
            JSONArray jsonArr =  object.getJSONObject("content").getJSONObject("positionResult").getJSONArray("result");

            parse(jsonArr);

            Thread.sleep(1000);
            if(i == 1) {
                Integer pageSize = object.getJSONObject("content").getInt("pageSize");
                Integer totalCount = object.getJSONObject("content").getJSONObject("positionResult").getInt("totalCount");
                System.out.println("totalCount" + totalCount);
                totalPage = (totalCount - 1) / pageSize;
                System.out.println("totalPage" + totalPage);
            }
            i++;
        }
    }

    public static  void parse(JSONArray positions)throws Exception{
        for(int i=1; i < positions.length(); i++){
            Position model = new Position();
            String education = positions.getJSONObject(i).getString("education");
            String jobNature = positions.getJSONObject(i).getString("education");
            String companyShortName = positions.getJSONObject(i).getString("companyShortName");
            String companyFullName = positions.getJSONObject(i).getString("companyFullName");

            String industryLables =positions.getJSONObject(i).get("industryLables").equals(JSONObject.NULL) ? null : join(positions.getJSONObject(i).getJSONArray("industryLables"));

            String businessZones = positions.getJSONObject(i).get("businessZones").equals(JSONObject.NULL) ? null : join(positions.getJSONObject(i).getJSONArray("businessZones"));

            String city = positions.getJSONObject(i).getString("city");
            String district = positions.getJSONObject(i).get("district").equals(JSONObject.NULL) ? null :  positions.getJSONObject(i).getString("district");
            Double latitude = positions.getJSONObject(i).get("latitude").equals(JSONObject.NULL) ? null :  positions.getJSONObject(i).getDouble("latitude");
            Double longitude = positions.getJSONObject(i).get("longitude").equals(JSONObject.NULL) ? null :  positions.getJSONObject(i).getDouble("longitude");
            String positionName = positions.getJSONObject(i).getString("positionName");
            String positionLables = positions.getJSONObject(i).get("positionLables").equals(JSONObject.NULL) ? null : join(positions.getJSONObject(i).getJSONArray("positionLables"));

            String firstType = positions.getJSONObject(i).getString("firstType");
            String secondType = positions.getJSONObject(i).getString("secondType");

            String companySize = positions.getJSONObject(i).get("companySize").equals(JSONObject.NULL) ? "0": positions.getJSONObject(i).getString("companySize") ;

            String stationName = positions.getJSONObject(i).get("stationname").equals(JSONObject.NULL) ? null : positions.getJSONObject(i).getString("stationname") ;
            String linestaion = positions.getJSONObject(i).get("linestaion").equals(JSONObject.NULL) ? null : positions.getJSONObject(i).getString("linestaion");
            String financeStage = positions.getJSONObject(i).get("financeStage").equals(JSONObject.NULL) ? null : positions.getJSONObject(i).getString("financeStage");
            String companyLabelList = positions.getJSONObject(i).get("companyLabelList").equals(JSONObject.NULL) ? null : join(positions.getJSONObject(i).getJSONArray("companyLabelList"));

            String industryField = positions.getJSONObject(i).get("industryField").equals(JSONObject.NULL) ? null : positions.getJSONObject(i).getString("industryField");
            String salary = positions.getJSONObject(i).getString("salary");
            String workYear = positions.getJSONObject(i).getString("workYear");
            String positionAdvantage = positions.getJSONObject(i).getString("positionAdvantage");
            String createTime = positions.getJSONObject(i).getString("createTime");

            model.setEducation(education);
            model.setJobNature(jobNature);
            model.setCompanyShortName(companyShortName);
            model.setCompanyFullName(companyFullName);
            model.setIndustryLables(industryLables);
            model.setBusinessZones(businessZones);
            model.setCity(city);
            model.setDistrict(district);
            model.setLatitude(latitude);
            model.setLongitude(longitude);
            model.setPositionName(positionName);
            model.setPositionLables(positionLables);
            model.setLatitude(latitude);
            model.setLongitude(longitude);
            model.setFirstType(firstType);
            model.setSecondType(secondType);
            model.setCompanySize(companySize);
            model.setStationName(stationName);
            model.setLinestaion(linestaion);
            model.setFinanceStage(financeStage);
            model.setCompanyLabelList(companyLabelList);
            model.setIndustryField(industryField);
            model.setSalary(salary);
            model.setWorkYear(workYear);
            model.setPositionAdvantage(positionAdvantage);

            model.setCreateTime(createTime);

            DataSet.insert(model);
        }
    }


    private static String join(JSONArray arr){
        StringBuilder stringBuilder = new StringBuilder();

        for(int i= 1; i < arr.length(); i++){
            stringBuilder.append(arr.get(i));
        }
        return stringBuilder.toString();
    }

    private static JSONObject getJsonObjectByPageIndex(Integer index){
        Map<String,String> headers = new HashMap<>();
        headers.put("Host","www.lagou.com");
        headers.put("Origin","https://www.lagou.com");
        headers.put("X-Anit-Forge-Code","0");
        headers.put("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
        headers.put("Content-Type","application/x-www-form-urlencoded; charset=UTF-8");
        headers.put("Accept","pplication/json, text/javascript, */*; q=0.01");
        headers.put("X-Requested-With","XMLHttpRequest");
        headers.put("X-Anit-Forge-Token","None");
        headers.put("Referer","https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98?labelWords=&fromSearch=true&suginput=");
        headers.put("Accept-Encoding","gzip, deflate, br");
        headers.put("Accept-Language","zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4");
        headers.put("Cookie","LGUID=20170212222937-ae7f6943-f12f-11e6-a685-525400f775ce; user_trace_token=20170212222936-7b4486912f87460dbe822d2c7014e7bb; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=8; index_location_city=%E6%B7%B1%E5%9C%B3; JSESSIONID=522C5ED68CC7FCFBE09ACB43565ECAD1; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1486909777,1486979194,1487125393; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1487299799; _gat=1; _ga=GA1.2.9339698.1486909777; LGSID=20170217104958-c562c41b-f4bb-11e6-8fc3-5254005c3644; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGRID=20170217104958-c562c67c-f4bb-11e6-8fc3-5254005c3644; TG-TRACK-CODE=index_search; SEARCH_ID=482ccd4314dd4b46a4cdd31db61f3cdf");

        JSONObject object = HttpRequestUtil.post("https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&city=上海&kd=java&pn="+index,headers);

        return object;
    }

}
